# -*- coding: utf-8 -*-
# 创建时间：2021/7/19 17:41
from bs4 import BeautifulSoup
import requests
import time, math, random, json
import pymysql
import re
from lxml import etree
from hhy import FileUtil,DateUtil,ToolUtil,HttpUtil
__author__ = 'LuckyHhy'


chinese_city_district_dict = dict()
chinese_area_dict = dict()


class SecHouse(object):
    def __init__(self, district, area, name, price, desc, pic):
        self.district = district
        self.area = area
        self.price = price
        self.name = name
        self.desc = desc
        self.pic = pic

    def text(self):
        return self.district + "," + \
               self.area + "," + \
               self.name + "," + \
               self.price + "," + \
               self.desc + "," + \
               self.pic


def create_headers():
    headers = dict()
    headers["User-Agent"] = HttpUtil.AgentRandom()
    headers["Referer"] = "http://www.ke.com"
    return headers


def get_districts():
    url = 'https://bj.ke.com/xiaoqu/'
    headers = create_headers()
    response = requests.get(url, timeout=10, headers=headers)
    html = response.content
    root = etree.HTML(html)
    elements = root.xpath('///div[3]/div[1]/dl[2]/dd/div/div/a')
    en_names = list()
    ch_names = list()
    for element in elements:
        link = element.attrib['href']
        en_names.append(link.split('/')[-2])
        ch_names.append(element.text)

    # 打印区县英文和中文名列表
    for index, name in enumerate(en_names):
        chinese_city_district_dict[name] = ch_names[index]
    return en_names



def get_areas(district):
    page = "http://bj.ke.com/xiaoqu/{0}".format(district)
    areas = list()
    try:
        headers = create_headers()
        response = requests.get(page, timeout=10, headers=headers)
        html = response.content
        root = etree.HTML(html)
        links = root.xpath('//div[3]/div[1]/dl[2]/dd/div/div[2]/a')

        # 针对a标签的list进行处理
        for link in links:
            relative_link = link.attrib['href']
            # 去掉最后的"/"
            relative_link = relative_link[:-1]
            # 获取最后一节
            area = relative_link.split("/")[-1]
            # 去掉区县名,防止重复
            if area != district:
                chinese_area = link.text
                chinese_area_dict[area] = chinese_area
                # print(chinese_area)
                areas.append(area)
        return areas
    except Exception as e:
        print(e)




def main():
    # districts = get_districts()
    # print(districts)
    # exit()
    with open("sechouse.txt", "w", encoding='utf-8') as f:
        # 开始获得需要的板块数据
        total_page = 1
        sec_house_list = list()
        # 获取小区
        districts = get_districts()

        for district in districts:
            arealist = get_areas(district)
            for area in arealist:
                # 中文区县
                chinese_district = chinese_city_district_dict.get(district, "")
                # 中文版块
                chinese_area = chinese_area_dict.get(area, "")
                page = 'http://bj.ke.com/ershoufang/{0}/'.format(area)
                print(page)
                headers = create_headers()
                response = requests.get(page, timeout=10, headers=headers)
                html = response.content
                soup = BeautifulSoup(html, "lxml")

                # 获得总的页数
                try:
                    page_box = soup.find_all('div', class_='page-box')[0]
                    matches = re.search('.*data-total-count="(\d+)".*', str(page_box))
                    total_page = int(math.ceil(int(matches.group(1)) / 10))
                except Exception as e:
                    print(e)

                print(total_page)
                # 从第一页开始,一直遍历到最后一页
                headers = create_headers()
                for i in range(1, total_page + 1):
                    page = 'http://bj.ke.com/ershoufang/{0}/pg{1}'.format(area, i)
                    print(page)
                    response = requests.get(page, timeout=10, headers=headers)
                    html = response.content
                    soup = BeautifulSoup(html, "lxml")

                    # 获得有小区信息的panel
                    house_elements = soup.find_all('li', class_="clear")
                    for house_elem in house_elements:
                        price = house_elem.find('div', class_="totalPrice")
                        name = house_elem.find('div', class_='title')
                        desc = house_elem.find('div', class_="houseInfo")
                        pic = house_elem.find('a', class_="img").find('img', class_="lj-lazy")

                        # 继续清理数据
                        price = price.text.strip()
                        name = name.text.replace("\n", "")
                        desc = desc.text.replace("\n", "").strip()
                        pic = pic.get('data-original').strip()

                        # 作为对象保存
                        sec_house = SecHouse(chinese_district, chinese_area, name, price, desc, pic)
                        print(sec_house.text())
                        sec_house_list.append(sec_house)

                for sec_house in sec_house_list:
                    f.write(sec_house.text() + "\n")


    pass


if __name__ == '__main__':
    main()
